{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "59110159-1ab6-4a9a-88a4-8b6dd3611ded",
   "metadata": {},
   "source": [
    "## Importing Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "063cabf1-0a29-46e4-b5f8-6523c244e98e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from transformers import BertTokenizer\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "sns.set(style=\"darkgrid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6cb7a179-911d-44cb-aa61-c249848b632b",
   "metadata": {},
   "source": [
    "## Importing the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3656f349-e5b2-491f-a5e8-2c916255de9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5010, 11)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>helpful_count</th>\n",
       "      <th>total_comments</th>\n",
       "      <th>url</th>\n",
       "      <th>review_country</th>\n",
       "      <th>reviewed_at</th>\n",
       "      <th>review_text</th>\n",
       "      <th>review_rating</th>\n",
       "      <th>product_company</th>\n",
       "      <th>profile_name</th>\n",
       "      <th>review_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apple iPhone XR (64GB) - Black</td>\n",
       "      <td>5,087 people found this helpful</td>\n",
       "      <td>24</td>\n",
       "      <td>https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...</td>\n",
       "      <td>India</td>\n",
       "      <td>2018-12-12</td>\n",
       "      <td>NOTE:</td>\n",
       "      <td>3.0 out of 5 stars</td>\n",
       "      <td>Apple</td>\n",
       "      <td>Sameer Patil</td>\n",
       "      <td>Which iPhone you should Purchase ? iPhone 8, X...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apple iPhone XR (64GB) - Black</td>\n",
       "      <td>2,822 people found this helpful</td>\n",
       "      <td>6</td>\n",
       "      <td>https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...</td>\n",
       "      <td>India</td>\n",
       "      <td>2018-11-17</td>\n",
       "      <td>Very bad experience with this iPhone xr phone....</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Apple</td>\n",
       "      <td>Amazon Customer</td>\n",
       "      <td>Don't buy iPhone xr from Amazon.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Apple iPhone XR (64GB) - Black</td>\n",
       "      <td>1,798 people found this helpful</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...</td>\n",
       "      <td>India</td>\n",
       "      <td>2019-01-27</td>\n",
       "      <td>Amazing phone with amazing camera coming from ...</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Apple</td>\n",
       "      <td>A</td>\n",
       "      <td>Happy with the purchase</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Apple iPhone XR (64GB) - Black</td>\n",
       "      <td>1,366 people found this helpful</td>\n",
       "      <td>14</td>\n",
       "      <td>https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...</td>\n",
       "      <td>India</td>\n",
       "      <td>2019-05-02</td>\n",
       "      <td>So I got the iPhone XR just today. The product...</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Apple</td>\n",
       "      <td>Shubham Dutta</td>\n",
       "      <td>Amazon is not an apple authorised reseller. Pl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Apple iPhone XR (64GB) - Black</td>\n",
       "      <td>536 people found this helpful</td>\n",
       "      <td>5</td>\n",
       "      <td>https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...</td>\n",
       "      <td>India</td>\n",
       "      <td>2019-05-24</td>\n",
       "      <td>I've been an android user all my life until I ...</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Apple</td>\n",
       "      <td>Nepuni Lokho</td>\n",
       "      <td>Excellent Battery life and buttery smooth UI</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          product                    helpful_count  \\\n",
       "0  Apple iPhone XR (64GB) - Black  5,087 people found this helpful   \n",
       "1  Apple iPhone XR (64GB) - Black  2,822 people found this helpful   \n",
       "2  Apple iPhone XR (64GB) - Black  1,798 people found this helpful   \n",
       "3  Apple iPhone XR (64GB) - Black  1,366 people found this helpful   \n",
       "4  Apple iPhone XR (64GB) - Black    536 people found this helpful   \n",
       "\n",
       "   total_comments                                                url  \\\n",
       "0              24  https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...   \n",
       "1               6  https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...   \n",
       "2               0  https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...   \n",
       "3              14  https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...   \n",
       "4               5  https://www.amazon.in/Apple-iPhone-XR-64GB-Bla...   \n",
       "\n",
       "  review_country reviewed_at  \\\n",
       "0         India   2018-12-12   \n",
       "1         India   2018-11-17   \n",
       "2         India   2019-01-27   \n",
       "3         India   2019-05-02   \n",
       "4         India   2019-05-24   \n",
       "\n",
       "                                         review_text       review_rating  \\\n",
       "0                                              NOTE:  3.0 out of 5 stars   \n",
       "1  Very bad experience with this iPhone xr phone....  1.0 out of 5 stars   \n",
       "2  Amazing phone with amazing camera coming from ...  5.0 out of 5 stars   \n",
       "3  So I got the iPhone XR just today. The product...  1.0 out of 5 stars   \n",
       "4  I've been an android user all my life until I ...  5.0 out of 5 stars   \n",
       "\n",
       "  product_company     profile_name  \\\n",
       "0           Apple     Sameer Patil   \n",
       "1           Apple  Amazon Customer   \n",
       "2           Apple                A   \n",
       "3           Apple    Shubham Dutta   \n",
       "4           Apple     Nepuni Lokho   \n",
       "\n",
       "                                        review_title  \n",
       "0  Which iPhone you should Purchase ? iPhone 8, X...  \n",
       "1                   Don't buy iPhone xr from Amazon.  \n",
       "2                            Happy with the purchase  \n",
       "3  Amazon is not an apple authorised reseller. Pl...  \n",
       "4       Excellent Battery life and buttery smooth UI  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = r\"D:\\\\Paper Dataset\\\\apple_iphone_11_reviews.json\"\n",
    "df = pd.read_json(file_path)\n",
    "\n",
    "# Display the first few rows of the DataFrame\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "002cb155-de90-483a-9c00-5e8d445ec14f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "review_rating\n",
       "5.0 out of 5 stars    3731\n",
       "4.0 out of 5 stars     720\n",
       "1.0 out of 5 stars     319\n",
       "3.0 out of 5 stars     153\n",
       "2.0 out of 5 stars      87\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"review_rating\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "258064b8-3029-4ade-af6a-1155df477936",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numeric_rating\n",
       "5.0    3731\n",
       "4.0     720\n",
       "1.0     319\n",
       "3.0     153\n",
       "2.0      87\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract the numeric rating and convert to float\n",
    "df['numeric_rating'] = df[\"review_rating\"].str.extract(r'([0-9]+\\.?[0-9]*)').astype(float)\n",
    "\n",
    "df['numeric_rating'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "94ce248d-f470-4241-83a0-078fa2f7b790",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "categorical_rating\n",
       "positive    4451\n",
       "negative     406\n",
       "neutral      153\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Function to convert ratings\n",
    "def convert_rating(rating):\n",
    "    if rating >= 4.0:\n",
    "        return 'positive'\n",
    "    elif rating == 3.0:\n",
    "        return 'neutral'\n",
    "    else:\n",
    "        return 'negative'\n",
    "\n",
    "# Apply the function to the rating column\n",
    "df['categorical_rating'] = df['numeric_rating'].apply(convert_rating)\n",
    "\n",
    "df['categorical_rating'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "750c0e11-b452-46c3-81be-da9b87657019",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAAHuCAYAAACVuKhMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABL10lEQVR4nO3df3zN9f//8fs5Zxv7YWyWDVEi5Fd+bahmKFJUflTe+ZEf+fFhfidR8pt6a42YicxvQjGFIpKo/OYdhT6hMMyw2bDN7Jzz/cN35/Pam7Kx7Wx2u14uLnZeP57ncc55zbl7Pp+v18tkt9vtAgAAgCTJ7OwCAAAA8hPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjALnqfr/OrLNfn7OfH7gfEY4ASJK6dOmiKlWqZPpTtWpV1a1bV+3atdOXX36Z7Tb37dun3r17Ox7HxMSoSpUqWr16dU6WflsZz2X8U6NGDT3xxBPq27ev9u/fn2n7Xbt2qUqVKtq1a1eW2k9LS9PkyZO1du3aO25bpUoVzZgx466e55848/0F7mcuzi4AQP5RrVo1jRkzxvHYarUqNjZWCxYs0PDhw1WiRAmFhIRkub3PP/9cx48fdzwuVaqUVqxYofLly+do3f+kb9++atKkiSTp+vXrio2N1eLFi9WpUyfNmDFDzzzzjCSpevXqWrFihSpVqpSlduPi4rRw4UK9//77d9x2xYoVCggIuOvX8Hfyw/sL3I8IRwAcvLy8VLt27VuWN27cWI0aNdLq1auzFY7+m5ub223bz03ly5e/5Tmfe+45de7cWe+++64aNmwoLy+vv33tOSGvXrMz3l/gfsSwGoA7KlKkiNzc3GQymRzL4uPjNW7cODVt2lQ1atRQUFCQQkNDFRMTI0kaMWKEoqOjdebMGcdQz38P+6xevVrVqlXTL7/8og4dOqhmzZpq2rSpoqKiMj1/XFychgwZoqCgIAUGBmr06NGaOnWqmjVrdlevx83NTQMGDNDly5f1zTffSLp1uCs1NVVjx45V48aNVaNGDbVs2dJRV0xMjJ5++mlJ0siRIx11jBgxQl27dtWYMWNUt25dPf/887JarZmG1TIcO3ZMHTt2VM2aNdW8eXMtXrzYse7vhsdGjBiR6bnu9P5K0l9//aWBAwfqySefVO3atdWlSxft27fvluf65ptvNHDgQNWpU0dBQUEaNWqUkpOT7+r9BQo6eo4AONjtdqWnpzseW61WnTlzRjNnztS1a9f00ksvObbr06ePEhMTNWzYMPn5+en333/XtGnTNGbMGEVFRalfv36Kj4/X4cOHFRERofLly9/2y9Zms2nw4MHq1q2bBg8erC+++EJTpkxR5cqVFRwcrLS0NHXt2lXJycl655135OXlpTlz5ujIkSN64IEH7vq1NmrUSGazWfv379crr7xyy/rJkyfrxx9/1Ntvvy0/Pz9t27ZNU6ZMUYkSJfTCCy8oIiJC/fv3V9++fdWiRQvHfnv37lWRIkU0c+ZMJScny2Kx3Pb533//fb3++uvq16+ftmzZookTJ8pms6lr165Zqj8r7++xY8f06quv6uGHH9aoUaPk6uqqRYsWqWvXrpo3b56CgoIc244ZM0bt27dXZGSkDh48qKlTp8rHx0dvvvlmluoB7ieEIwAOe/bsUfXq1TMtM5lMqly5sj7++GM1bdpU0s2eHHd3d7399tuqX7++JKlBgwY6deqUVqxYIenmcJavr2+moZ7bhSO73a5+/fo5Akq9evW0adMmbd26VcHBwfrqq6904sQJrVq1SjVq1JAkNWzY0DFX6G65uLjIx8dHFy5cuO363bt368knn1SrVq0cr8/Dw0MlS5aUm5ubHnvsMcfrrFatmmO/9PR0jR8//o5zjF599VUNHz5ckvTUU0/p/Pnzmj17trp06ZKl+rPy/kZERMjNzU2LFi2Sl5eXJKlJkyZq3bq1pkyZoi+++MKxbUhIiN5++21JN4PjTz/9pK1btxKOUCgRjgA4VK9eXePGjZN0MwBNmzZNN27c0LRp0/TII484tvP399eiRYtkt9sVExOjkydP6sSJE9q/f7/S0tKy/bx16tRx/Ozm5iZfX1/HF/3OnTtVrlw5RzCSbs6Natq06T2f8WW32zMNFRo1aNBAy5cvV2xsrEJCQhQSEqLQ0NA7tlmiRIksTb5+/vnnMz1u3ry5Nm/erBMnTqho0aJZewF3sHv3bjVt2tQRjKSbobBVq1aO3sAM/z1XKSAgQGfOnMmROoCChnAEwMHT01M1a9Z0PH788cf14osvqkePHlq9erV8fX0d67766iuFh4fr3LlzKlGihB577LG7/lL/7/3MZrPj+j0JCQkqWbLkLfvcbll2pKSkKDEx8W+DzLvvvquAgAB99dVXmjBhgiZMmKA6depo7Nixqlq16t+26+npmaXn9/Pzy/Q44/UkJibmWDhKTEy85Xkynttut+vq1auOZe7u7pm2MX4GQGHDhGwAf8vPz0+jR4/WuXPnNGnSJMfyvXv36u2331aLFi20bds27dq1SwsWLMiVM6X8/f118eLFW5ZfunTpntrdvXu3rFarAgMDb7vezc1Nffv21TfffKPvv/9eo0eP1unTp3NsmCkxMTHT44zXWLJkSUdvltVqzbRNdidIFy9e/LbvXcZQoo+PT7baAwoLwhGAf9SyZUsFBwdr3bp12r17tyTpwIEDstlsGjBggPz9/SXd/CL/+eefJd2cZC3d7H24V0FBQYqJidGRI0ccy1JTU7V9+/a7bjM9PV2RkZHy8/NT8+bNb1mfmpqqZ599VvPmzZMklSlTRp06dVKrVq109uxZSfrbidZZtXXr1kyP169fr9KlS+uhhx5yDIOdP3/esf7GjRs6ePBgpn3u9P4GBgbq+++/z9RDZLVatX79etWsWVNubm739BqA+xXDagDu6J133tGLL76oiRMnKjo6WrVq1ZIkjR8/Xu3bt1diYqKWLl2qo0ePSrrZw+Hl5SVvb29dvHhRP/zwg2MCc3a1bt1ac+bMUWhoqAYNGiRvb2/Nnz9fly5dUpkyZe64/6lTp/Sf//xH0s2AERMTo+XLl+u3337TzJkzbxlOkm4O81WvXl0RERFydXVVlSpV9Oeffyo6OlrPPvusJKlYsWKSpB07dqhixYp6/PHHs/W6Fi9eLE9PT1WrVk3r16/X9u3bNWXKFJlMJhUvXlx16tTR4sWL9dBDD6l48eJatGiRUlNT5eHh4WjjTu9v//79tW3bNr3++uvq3bu3XF1dtWTJEp0+fVpz587NVr1AYULPEYA7euSRR9SlSxf9/vvv+uyzz9SgQQONHj1aBw4cUK9evfTBBx+oTJkyioiIkCTHdXTatWunsmXLKjQ0VGvWrLmr53ZxcVFUVJSqVaumsWPHavjw4Xr00UfVvHnzTEHh78yaNUsdOnRQhw4d1K1bN4WHh6tMmTJasWLFP17Qcvz48WrXrp3mzZunHj16KDIyUi+//LLGjh0r6eak8O7du2vz5s3q1auXbty4ka3XNXHiRG3YsEG9e/fW/v37FR4e7rhUgiR98MEHqlGjhkaNGqWRI0eqevXqt5zmf6f399FHH9WyZctUsmRJjRw5Um+99ZbsdrsWLVqkJ554Ilv1AoWJyc6MOwD52B9//KETJ06oRYsWmc4se/nllxUQEOAIZACQUxhWA5CvJScna9CgQerYsaOaN28uq9Wqr7/+Wr/++quGDRvm7PIA3IfoOQKQ723YsEFRUVE6fvy47Ha7qlWrpr59++qpp55ydmkA7kOEIwAAAAMmZAMAABgQjgAAAAwIRwAAAAaEIwAAAANO5b9LdrtdNhtz2e+V2WzifUS+w3GJ/IZjMmeYzaZM10v7O4Sju2Sz2RUff83ZZRRoLi5m+fh4KikpWenpNmeXA0jiuET+wzGZc3x9PWWx3DkcMawGAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAGhCMAAAADwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgIGLswtA7jGbTTKbTc4u429ZLOZMf+dnNptdNpvd2WUAAPIA4eg+ZTabVKKER4EIHt7e7s4u4Y6sVpsuX04mIAFAIUA4uk+ZzSZZLGaFLd2nmPNXnF1OgfagfzEN61RPZrOJcAQAhQDh6D4Xc/6Kjp9JdHYZAAAUGPl/zAUAACAPEY4AAAAMCEcAAAAGhCMAAAADwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAAeEIAADAgHAEAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAG+Soc/fnnn6pTp45Wr17tWHbkyBF17txZtWvXVrNmzbRo0aJM+9hsNk2fPl3BwcGqXbu2evXqpdOnT2fa5k5tAAAAZMg34ejGjRsaNmyYkpOTHcsSEhLUvXt3lS9fXqtWrVJoaKjCwsK0atUqxzaRkZFatmyZJkyYoOXLl8tms6lnz55KS0vLchsAAAAZXJxdQIYZM2bIy8sr07KVK1fK1dVV48ePl4uLiypWrKiTJ09qzpw5at++vdLS0jRv3jwNGzZMTZo0kSRNnTpVwcHB+vbbb9W6des7tgEAAGCUL3qO9uzZoxUrVuiDDz7ItHzv3r0KCgqSi8v/ZbiGDRvqr7/+0sWLF3X06FFdu3ZNjRo1cqz39vZWtWrVtGfPniy1AQAAYOT0nqOkpCQNHz5co0aNUunSpTOti42NVeXKlTMtK1WqlCTp3Llzio2NlaRb9itVqpRj3Z3a8PPzu+vaXVzyRba8LYsl/9ZWUPGeFg4ZnzOfN/ILjsm85/RwNHbsWNWpU0cvvPDCLetSU1Pl5uaWaVmRIkUkSdevX1dKSook3XabxMTELLVxt8xmk3x8PO96fxQ83t7uzi4BeYjPG/kNx2TecWo4WrNmjfbu3au1a9fedn3RokUdE6szZAQaDw8PFS1aVJKUlpbm+DljG3d39yy1cbdsNruSkpLvvKGTWCxmfpFyWFJSiqxWm7PLQC7L+N3h80Z+wTGZc7y93bPUA+fUcLRq1SpdunTJMZk6w5gxY/T1118rICBAcXFxmdZlPPb391d6erpjWfny5TNtU6VKFUm6Yxv3Ij2dg7QwsVptfOaFCJ838huOybzj1HAUFham1NTUTMtatGihgQMH6sUXX9SXX36p5cuXy2q1ymKxSJJ27typChUqqGTJkipWrJi8vLy0a9cuRzhKSkrS4cOH1blzZ0lSYGDgP7YBAABg5NTZXf7+/nrooYcy/ZGkkiVLyt/fX+3bt9fVq1f17rvv6tixY1q9erUWLFigPn36SLo516hz584KCwvTd999p6NHj2rIkCEKCAhQixYtJOmObQAAABg5fUL2PylZsqTmzp2rSZMmqW3btnrggQc0fPhwtW3b1rHNwIEDlZ6erlGjRik1NVWBgYGKioqSq6trltsAAADIYLLb7XZnF1EQWa02xcdfc3YZf8vFxSwfH08NDt+q42cSnV1OgVaxbHFNG9pECQnXGO8vBDJ+d/i8kV9wTOYcX1/PLE3I5qIJAAAABoQjAAAAA8IRAACAAeEIAADAgHAEAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAGhCMAAAADwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAAeEIAADAgHAEAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAGhCMAAAADwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAAeEIAADAgHAEAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAGhCMAAAADwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgIHTw9GlS5f01ltvqWHDhqpTp4569+6t48ePO9YfOXJEnTt3Vu3atdWsWTMtWrQo0/42m03Tp09XcHCwateurV69eun06dOZtrlTGwAAABmcHo5CQ0N18uRJzZkzR1988YWKFi2qbt26KSUlRQkJCerevbvKly+vVatWKTQ0VGFhYVq1apVj/8jISC1btkwTJkzQ8uXLZbPZ1LNnT6WlpUlSltoAAADI4OLMJ09MTFTZsmXVp08fVa5cWZLUr18/vfTSS/rjjz+0Y8cOubq6avz48XJxcVHFihUdQap9+/ZKS0vTvHnzNGzYMDVp0kSSNHXqVAUHB+vbb79V69attXLlyn9sAwAAwChHeo7S09N1+fLlbO9XvHhxffTRR45gFB8frwULFiggIECVKlXS3r17FRQUJBeX/8twDRs21F9//aWLFy/q6NGjunbtmho1auRY7+3trWrVqmnPnj2SdMc2AAAAjLLdc5Senq5PPvlEDz30kF544QXt2rVLAwcOVFJSkoKCgjR9+nQVL14824W89957Wrlypdzc3DRr1ix5eHgoNjbWEZwylCpVSpJ07tw5xcbGSpJKly59yzYZ6+7Uhp+fX7ZrzeDi4vRRyb9lseTf2goq3tPCIeNz5vNGfsExmfeyHY6mT5+uqKgovfPOO5KkiRMnqkSJEgoNDdX8+fP10Ucfafz48dkupGvXrurQoYOWLl2q0NBQLVu2TKmpqXJzc8u0XZEiRSRJ169fV0pKiiTddpvExERJumMbd8tsNsnHx/Ou90fB4+3t7uwSkIf4vJHfcEzmnWyHo/Xr12vo0KHq1KmTjh8/rj/++EMffPCB2rRpoxIlSmjKlCl3FY4qVaokSZo0aZJ++eUXLVmyREWLFnVMrM6QEWg8PDxUtGhRSVJaWprj54xt3N1vHkR3auNu2Wx2JSUl3/X+uc1iMfOLlMOSklJktdqcXQZyWcbvDp838guOyZzj7e2epR64bIejuLg4Pf7445KkrVu3ymw2q3HjxpKkgIAAXblyJcttxcfHa8eOHXr22Wcdc4LMZrMqVaqkuLg4BQQEKC4u7pbnlyR/f3+lp6c7lpUvXz7TNlWqVHHU9E9t3Iv0dA7SwsRqtfGZFyJ83shvOCbzTrYHMEuVKqWYmBhJ0pYtW/TYY4/J19dXknTgwAEFBARkua2LFy9q6NCh2rFjh2PZjRs3dPjwYVWsWFGBgYHat2+frFarY/3OnTtVoUIFlSxZUlWrVpWXl5d27drlWJ+UlKTDhw8rMDBQku7YBgAAgFG2w1Hr1q31/vvv64033tC+ffscp8NPmjRJM2bM0AsvvJDltipXrqzGjRtr4sSJ2rNnj/73f/9XI0aMUFJSkrp166b27dvr6tWrevfdd3Xs2DGtXr1aCxYsUJ8+fSTdnGvUuXNnhYWF6bvvvtPRo0c1ZMgQBQQEqEWLFpJ0xzYAAACMsj2sNnjwYHl4eGjPnj1688031bFjR0nSoUOH1KNHD/Xr1y9b7YWHh+ujjz7SkCFDdOXKFdWvX19Lly5VmTJlJElz587VpEmT1LZtWz3wwAMaPny42rZt69h/4MCBSk9P16hRo5SamqrAwEBFRUXJ1dVVklSyZMk7tgEAAJDBZLfb7dnZ4ZdffnHMOSrMrFab4uOvObuMv+XiYpaPj6cGh2/V8TOJzi6nQKtYtrimDW2ihIRrjPcXAhm/O3zeyC84JnOOr69n7kzI7tChg0qWLKmQkBA1bdpUTz755D2d9QUAAJCfZDscRUdHa9u2bfrxxx81ZMgQmc1mBQYGqmnTpmrSpIkefPDB3KgTAAAgT2Q7HD322GN67LHH1KdPH129elU7duzQtm3bFBUVpUmTJqlSpUpau3ZtbtQKAACQ6+7pWuRXrlxRcnKyrFarzGaz7Ha747YdAAAABVG2e45WrVqlvXv3avfu3Tpz5ozc3d1Vt25d/etf/1KDBg1Uo0aN3KgTAAAgT2Q7HL377rsymUyqXr26Ro4cqSZNmmS64z0AAEBBlu1U88EHH2jXrl3asWOHBgwYoAoVKigoKEhBQUFq0KABV50GAAAFWrbDUZs2bdSmTRtJ0p9//qmdO3dq586dmjx5si5duqRHHnlE69evz+k6AQAA8sQ9Tch+8MEHVbFiRVWoUEEPPvig7Ha7Lly4kFO1AQAA5Lls9xwdPXpUP//8s37++Wft27dPqampqlChgkJCQjRkyBDVq1cvN+oEAADIE3c1rObm5qagoCC9+eabCgkJUbly5XKjNgAAgDyX7XAUGRmpRo0ayd3dPTfqAQAAcKpsh6NmzZopLS1Ny5Yt088//6wLFy5o8uTJ2r17t6pXr65atWrlRp0AAAB5ItsTsuPj49W+fXtNmjRJJ0+e1MGDB5WamqqtW7eqS5cuOnDgQG7UCQAAkCeyHY6mTJmia9eu6euvv1Z0dLTsdrskafr06apZs6amT5+e40UCAADklWyHo++//16DBg3SQw89JJPJ5FhepEgR9ejRQ7/99luOFggAAJCXsh2Orl+/rhIlStx2ncVi0Y0bN+61JgAAAKfJdjiqWbOmli1bdtt1a9eu5cazAACgQMv22WqDBg1St27d9NJLLykkJEQmk0nr1q3TjBkz9OOPP2ru3Lm5UScAAECeyHbPUf369TV//ny5u7tr7ty5stvtWrBggS5cuKDZs2erYcOGuVEnAABAnsh2z5EkBQYGavny5UpNTVViYqK8vLzk6emZ07UBAADkuSyFo7Nnz+qBBx6Qq6urzp49e8v6xMREJSYmOh6XKVMm5yoEAADIQ1kKR08//bRWrFihWrVqqVmzZplO4b+dI0eO5EhxAAAAeS1L4Wjy5MmOm8tOnjz5juEIAACgoMpSOGrbtq3j5yZNmsjX1zfXCgIAAHCmbJ+t1rhxY/Xt21cbNmxQWlpabtQEAADgNNkOR8OGDdOlS5c0ePBgPfnkkxo1apT27t2bG7UBAADkuWyfyt+tWzd169ZNp0+f1rp16/T111/riy++UJkyZfTiiy/qhRdeUMWKFXOjVgAAgFyX7Z6jDOXKlVPfvn21du1arV27Vk2aNNGnn36q1q1b52R9AAAAeequLgKZ4dKlS/rmm2/0zTff6MCBAypRooSef/75nKoNAAAgz2U7HF25ckUbN27U+vXrtWfPHlksFjVr1kyRkZEKDg6WxWLJjToBAADyRLbDUaNGjWSz2VSvXj2NHTtWLVu2lJeXV27UBgAAkOeyHY4GDBigF154gVuEAACA+1K2w1GfPn0k3byf2t69exUXF6dnn31Wly9fVoUKFbh6NgAAKNDuakL2rFmzNHv2bKWmpspkMqlWrVqaNm2aEhISNG/ePHl7e+d0nQAAAHki26fyL1myRDNmzFD37t21cuVK2e12SVLnzp11+vRpffzxxzleJAAAQF7JdjhavHixevfurUGDBql69eqO5SEhIRo8eLC2bNmSowUCAADkpWyHo7NnzyooKOi26x555BFdvHjxnosCAABwlmyHo9KlS+vAgQO3Xffrr7+qdOnS91wUAACAs2R7QvbLL7+sGTNmqGjRomrSpIkkKTk5WRs3btTs2bPVvXv3nK4RAAAgz2Q7HPXq1UsxMTEKCwtTWFiYJOn111+XJL3wwguOU/0BAAAKomyHI5PJpPHjx6tHjx7auXOnLl++rGLFiikwMFCVK1fOjRoBAADyzF3fePbhhx/Www8/nGmZ3W7XsmXL1KlTp3utCwAAwCmyHI62bdum6OhomUwmvfTSSwoJCcm0fu/evZo4caJ+//13whEAACiwshSOvvrqKw0fPlyurq5yc3PTN998o+nTp6t58+a6fPmyJk6cqPXr18tisTAhGwAAFGhZCkcLFy7U448/rqioKLm5uWnkyJGaOXOmHn30UXXv3l3nzp1TcHCw3nnnHVWoUCG3awYAAMg1WQpHf/31lyZMmCAvLy9JUv/+/fX888+rX79+SktL08cff6xnn302VwsFAADIC1kKR8nJyZku7li2bFnZ7Xa5uLjoq6++UsmSJXOtQAAAgLyUpStk2+12WSwWx+OMn4cMGUIwAgAA95Vs3z7EqFSpUjlVBwAAQL5wT+HIZDLlVB0AAAD5QpavczR27FjHhGy73S5Jeu+99+Tp6ZlpO5PJpIULF+ZgiQAAAHknS+EoMDBQ0v+For9bdrvHAAAABUmWwtHixYtzuw4AAIB84Z7mHAEAANxvCEcAAAAGhCMAAAADwhEAAIBBlsJR79699ccff0iS9uzZo2vXruVqUQAAAM6SpXC0Y8cOXbp0SZL0+uuv6/jx47laFAAAgLNk6VT+MmXKaMyYMapbt67sdrsiIyPl4+Nz221NJpMmT56co0UCAADklSyFo/Hjx2vKlCnavXu3TCaTfv31V7m5ud12W24pAgAACrIshaMGDRpo1apVkqSqVasqMjJStWrVytXCAAAAnCHL91bL8N1336lUqVKSpJSUFF29elUlSpSQq6trjhcHAACQ17IdjsqWLau9e/dqypQp+vXXXx33UqtVq5aGDBmihg0b5niRAAAAeSXb4Wj//v3q1q2bypUrp379+snPz09xcXFav369evbsqcWLF6tOnTq5USsAAECuy3Y4mjZtmurXr6+oqChZLBbH8v79++uNN97QjBkzNG/evBwtEgAAIK9k+wrZhw4d0uuvv54pGEmS2WxW586ddfDgwWy1d/nyZY0ePVqNGzdW3bp19dprr2nv3r2O9Tt27FC7du30+OOPq2XLllq/fn2m/a9fv65x48apUaNGqlOnjt58803Fx8dn2uZObQAAAGTIdjjy9PRUenr6bdelp6c75iBl1dChQ3XgwAGFh4dr1apVeuyxx/TGG2/oxIkTOn78uPr06aPg4GCtXr1ar7zyioYPH64dO3Y49h87dqx+/PFHzZgxQwsXLtSJEyc0cOBAx/qstAEAAJAh28NqdevW1Zw5cxQcHCx3d3fH8uTkZM2ZM0f169fPclsnT57UTz/9pGXLlqlevXqSpPfee0/bt2/X2rVrdenSJVWpUkVDhgyRJFWsWFGHDx/W3Llz1ahRI50/f15r1qzRJ5984nje8PBwtWzZUgcOHFCdOnW0cOHCf2wDAADAKNvh6M0331S7du309NNPq0mTJnrggQd04cIFbd26VampqZo0aVKW2/Lx8dGcOXNUs2ZNxzKTySSTyaSkpCTt3btXzzzzTKZ9GjZsqEmTJslut2vfvn2OZRkqVKggf39/7dmzR3Xq1LljG1y0EgAAGGU7HD300ENauXKlZsyYoR9++EGJiYkqXry4goKC1L9/f1WqVCnLbXl7eyskJCTTso0bN+rkyZN65513FB0drYCAgEzrS5UqpZSUFCUkJOj8+fPy8fFRkSJFbtkmNjZWkhQbG/uPbfj6+mbn5Wfi4pLtUck8Y7Hk39oKKt7TwiHjc+bzRn7BMZn3sh2OpJtDU9OmTcvhUm5eJmDkyJFq0aKFmjRpotTU1FtuU5LxOC0tTSkpKbe9jUmRIkV0/fp1SbpjG3fLbDbJx8fzrvdHwePt7X7njXDf4PNGfsMxmXfuKhzlhs2bN2vYsGGqW7euwsLCJN0MOf8dYDIeu7u7q2jRorcNONevX3fMh7pTG3fLZrMrKSn5rvfPbRaLmV+kHJaUlCKr1ebsMpDLMn53+LyRX3BM5hxvb/cs9cDli3C0ZMkSTZo0SS1bttS///1vR89O6dKlFRcXl2nbuLg4eXh4qFixYgoICNDly5eVlpaWqXcoLi5O/v7+WWrjXqSnc5AWJlarjc+8EOHzRn7DMZl3nD6AuWzZMk2YMEGdOnVSeHh4ppBTv3597d69O9P2O3fuVN26dWU2m1WvXj3ZbDbHxGxJ+vPPP3X+/HkFBgZmqQ0AAAAjp6aDP//8U5MnT1bz5s3Vp08fXbx4URcuXNCFCxd05coVdenSRQcPHlRYWJiOHz+uefPmacOGDerZs6ckyd/fX61atdKoUaO0a9cuHTx4UEOHDlVQUJBq164tSXdsAwAAwMipw2obN27UjRs3tGnTJm3atCnTurZt2+qDDz5QZGSkPvzwQy1cuFAPPvigPvzww0zXJ5owYYImT56s/v37S5IaN26sUaNGOdY/+uijd2wDAAAgg8me3Uta/4P4+Hht27ZNbdq0yakm8y2r1ab4+GvOLuNvubiY5ePjqcHhW3X8TKKzyynQKpYtrmlDmygh4Rrj/YVAxu8OnzfyC47JnOPr65mlCdk5Oqx2+vRpjRw5MiebBAAAyFM5Go4qVqyohQsX5mSTAAAAeSpHw5GXl5eCgoJyskkAAIA8ddcTsrdt26bdu3crKSlJPj4+ql+/voKDg3OyNgAAgDyX7XCUlpamfv366ccff5TFYpGPj48SEhI0Z84cNWzYULNnz77tLT0AAAAKgmwPq82YMUP79u3TlClTdPDgQf3444/65Zdf9P777+s///mPZs2alRt1AgAA5Ilsh6N169apf//+evHFF2WxWCRJLi4uatOmjfr376+1a9fmeJEAAAB5JdvhKD4+XtWqVbvtumrVqun8+fP3XBQAAICzZDsclS9fPtO9zIz27Nmj0qVL33NRAAAAzpLtCdn/+te/9MEHH6ho0aJq1aqV/Pz8dPHiRa1bt06ffvqp4zYeAAAABVG2w9Frr72mw4cPKywsTB999JFjud1uV9u2bdW7d+8cLRAAACAvZTscmc1mTZo0Sd27d3dc56h48eIKCgpSxYoVc6NGAACAPHPXF4GsVKmSKlWqlJO1AAAAOF2WwlF2biZrMpk0efLkuy4IAADAmbIUjnbt2nXHbRISEpSSkkI4AgAABVqWwtGWLVv+dl16eroiIyM1Z84c+fn5aezYsTlVGwAAQJ676zlHknTkyBGNHDlSv//+u1q1aqX33ntPxYsXz6naAAAA8txdhaP09HTNnDlTn376qUqUKKGIiAg9/fTTOV0bAABAnst2ODp8+LCjt+jFF1/UqFGj5O3tnRu1AQAA5Lksh6P09HRFRERo7ty58vHx0axZs9S0adPcrA0AACDPZSkc/fbbbxoxYoSOHTumNm3a6J133lGxYsVyuzYAAIA8l6Vw9Oqrr8pms6lYsWI6c+aMQkND/3Zbk8mkhQsX5liBAAAAeSlL4ahu3bqOn+12+z9ue6f1AAAA+VmWwtHixYtzuw4AAIB8wezsAgAAAPITwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAAeEIAADAgHAEAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAGhCMAAAADwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAAeEIAADAIF+Fo9mzZ6tLly6Zlh05ckSdO3dW7dq11axZMy1atCjTepvNpunTpys4OFi1a9dWr169dPr06Wy1AQAAkCHfhKOlS5dq2rRpmZYlJCSoe/fuKl++vFatWqXQ0FCFhYVp1apVjm0iIyO1bNkyTZgwQcuXL5fNZlPPnj2VlpaW5TYAAAAyuDi7gPPnz2vMmDHatWuXHn744UzrVq5cKVdXV40fP14uLi6qWLGiTp48qTlz5qh9+/ZKS0vTvHnzNGzYMDVp0kSSNHXqVAUHB+vbb79V69at79gGAACAkdN7jn777Te5urrqq6++0uOPP55p3d69exUUFCQXl//LcA0bNtRff/2lixcv6ujRo7p27ZoaNWrkWO/t7a1q1appz549WWoDAADAyOk9R82aNVOzZs1uuy42NlaVK1fOtKxUqVKSpHPnzik2NlaSVLp06Vu2yVh3pzb8/PzuunYXF6dny79lseTf2goq3tPCIeNz5vNGfsExmfecHo7+SWpqqtzc3DItK1KkiCTp+vXrSklJkaTbbpOYmJilNu6W2WySj4/nXe+Pgsfb293ZJSAP8Xkjv+GYzDv5OhwVLVrUMbE6Q0ag8fDwUNGiRSVJaWlpjp8ztnF3d89SG3fLZrMrKSn5rvfPbRaLmV+kHJaUlCKr1ebsMpDLMn53+LyRX3BM5hxvb/cs9cDl63AUEBCguLi4TMsyHvv7+ys9Pd2xrHz58pm2qVKlSpbauBfp6RykhYnVauMzL0T4vJHfcEzmnXw9gBkYGKh9+/bJarU6lu3cuVMVKlRQyZIlVbVqVXl5eWnXrl2O9UlJSTp8+LACAwOz1AYAAIBRvg5H7du319WrV/Xuu+/q2LFjWr16tRYsWKA+ffpIujnXqHPnzgoLC9N3332no0ePasiQIQoICFCLFi2y1AYAAIBRvh5WK1mypObOnatJkyapbdu2euCBBzR8+HC1bdvWsc3AgQOVnp6uUaNGKTU1VYGBgYqKipKrq2uW2wAAAMhgstvtdmcXURBZrTbFx19zdhl/y8XFLB8fTw0O36rjZxKdXU6BVrFscU0b2kQJCdcY7y8EMn53+LyRX3BM5hxfX88sTcjO18NqAAAAeY1wBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAAeEIAADAgHAEAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAGhCMAAAADwhEAAIAB4QgAAMCAcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAAeEIAADAgHAEAABgQDgCAAAwIBwBAAAYEI4AAAAMCEcAAAAGhCMAAAADF2cXAKDwMJtNMptNzi7jH1ks5kx/51c2m102m93ZZQD3JcIRgDxhNptUooRHvg8dGby93Z1dwj+yWm26fDmZgATkAsIRgDxhNptksZgVtnSfYs5fcXY5BdqD/sU0rFM9mc0mwhGQCwhHAPJUzPkrOn4m0dllAMDfKhj92wAAAHmEcAQAAGBAOAIAADAgHAEAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGBCOAAAADAhHAAAABoQjAAAAA8IRAACAgYuzCwAAwJnMZpPMZpOzy/hbFos509/5lc1ml81md3YZOYJwBAAotMxmk0qU8Mj3wUOSvL3dnV3CP7Jabbp8Ofm+CEiEIwBAoWU2m2SxmBW2dJ9izl9xdjkF1oP+xTSsUz2ZzSbCEQAA94OY81d0/Eyis8tAPpH/+xEBAADyEOEIAADAgHAEAABgQDgCAAAwIBwBAAAYFJpwZLPZNH36dAUHB6t27drq1auXTp8+7eyyAABAPlNowlFkZKSWLVumCRMmaPny5bLZbOrZs6fS0tKcXRoAAMhHCkU4SktL07x58zRw4EA1adJEVatW1dSpUxUbG6tvv/3W2eUBAIB8pFCEo6NHj+ratWtq1KiRY5m3t7eqVaumPXv2OLEyAACQ3xSKK2THxsZKkkqXLp1pealSpRzrsstsNsnX1/Oea8stpv9/D8WxvRop3WpzbjEFnMv/v+dS8eLushf8q+I7DcdkzuGYzDkclzmjoByTWb3BcKEIRykpKZIkNze3TMuLFCmixMS7u1y8yWSSxZJ/7+KcoUSxIs4u4b5hNheKjtZcxzGZczgmcw7HZc64X47J++NV3EHRokUl6ZbJ19evX5e7e/6+yzEAAMhbhSIcZQynxcXFZVoeFxcnf39/Z5QEAADyqUIRjqpWrSovLy/t2rXLsSwpKUmHDx9WYGCgEysDAAD5TaGYc+Tm5qbOnTsrLCxMvr6+Klu2rD788EMFBASoRYsWzi4PAADkI4UiHEnSwIEDlZ6erlGjRik1NVWBgYGKioqSq6urs0sDAAD5iMluz88n3QEAAOStQjHnCAAAIKsIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQBwn7h69aqzS7gvEI4AALgPrFy5UuPGjdO5c+ecXUqBRzgCAOA+cOrUKR06dEiffvqpYmNjnV1OgVZo7q2Ggs9ms8lsvjXP2+12mUwmJ1QE3PR3xyaQl4YNGyZPT0+tX79eVqtVffv2VUBAgLPLKpC4txoKBOOXz759+xQbGyur1aqqVauqcuXKBCQ4jfHY3Lx5s2JiYpSQkKCqVavqueeec3J1KCysVqssFoskKSIiQhs2bFC9evUISHeJcIQCJSwsTJs2bVLRokXl4eGhAwcOaM6cOWrcuLGzS0Mh9+GHH2rt2rWqU6eOkpKStG/fPrVv315jxoxxdmm4z2X859AYkGbMmKENGzaofv36BKS7QD8wCozly5crOjpakyZN0pdffun4X7nValVSUpKTq0Nhtn79eq1du1YzZszQxx9/rHbt2slsNuuJJ57Q6dOnnV0e7mMZwejQoUOKjo5WdHS0JGnAgAF67rnntHfvXs2aNYs5SNlEOEK+ldGpabVaZbfbdejQIb3yyiuqX7++Nm/erKlTp2ry5Ml68MEHNXHiRH75kWdsNlumv0+cOKG6devq8ccf14YNGzRmzBiNHDlSVapU0UcffaSDBw86s1zcpzKC0YYNG9StWzfNnj1bI0eO1MiRIyVJ/fv3dwSkOXPm6MyZM06uuOAgHCHfyphDZLVaZTKZlJCQoJIlS2rr1q166623NHz4cLVr106nT5/Wli1bdOnSJSdXjMIiY47RsWPHJEkJCQny9fXVzz//rJEjR+qtt95Shw4ddOPGDW3atElnz551Zrm4T5lMJv34448aOXKk3n77ba1evVrvvfeeoqOjNXr0aEk3A1KrVq307bffavHixbJarU6uumDgbDXka59//rnWrFmjJUuWqEyZMpo2bZpsNptGjhypV199VZJUsmRJ+fv7q2jRok6uFoXJ5s2bFR4erhUrVqhBgwYaNGiQli1bpo8++kitWrWSJLm6uqpixYry8/NzcrW432T0rH/99dfq0qWLXn31VcXHx2vdunVq0qSJ1q1bpxs3buj9999Xv3795OHhoWbNmjnmJOGf0XOEfMtms+natWtKSEhQQkKC3nrrLT322GPy8PBQnTp1dOnSJcXHx2vGjBkqVaqUKlSo4OySUYhYLBbFxsbq8OHDevbZZ9WtWzdZLBalpaXpzz//1F9//aWJEyfK3d1ddevWdXa5uE9khKK4uDiZTCb9+uuvjnVz5szRo48+qn//+9/q2bOnoqOj1bt3bx09elTdunVT+fLlnVV2gcPZasg30tPT5eKSuTPz6tWreu655/Tss89q1KhROnr0qEaMGKGzZ8/K09NTPj4+kqQVK1bI1dWV680gV9zu2JSkN998U6dOndLixYuVkJCghQsXasmSJfL09FTJkiVVvHhxLVq0SK6urpnOJALuxaZNmxQeHq6pU6fqxIkTcnNzk6+vr+bOnat27drpmWee0fz587Vq1SpZLBZFRESoXLlyzi67QCEcwel2796toKAgx+P4+Hj5+vo6Hn/++edavHixPvzwQ1WpUkXSzcvk37hxQyVKlFDLli1lsVj+9gsMuFvffPONQkJC5OHhIUk6ffq0fH195enpKUnaunWrpk2bpnfffVeBgYGSpF9//VUXLlxQsWLFVLduXZnNZo5N5Jhjx45p1qxZatiwoV555RUlJyfL3d1dU6ZM0b59+7Ry5UpJ0vjx4+Xv76/u3bvLzc3NyVUXPPwXG041duxYrVmzxnHWz4IFC9SxY0dNmTLFcQPFoKAgJScnZzrj59VXX1WnTp3UqlUrWSwWWa1WvnyQo6ZPn67o6GjHXLbNmzerefPmGjt2rL777jtJUnBwsLy8vBQVFeXYr0aNGmratKnq168vs9nMsYm7dv78eSUkJDgenz59WqGhodq1a5djiMzDw0Mmk0nFihWTyWTSmjVr9OGHH2rDhg16+umnCUZ3iXAEp+rQoYPGjRsns9mss2fPqnnz5nrqqae0fft2Pffcc5o5c6Y8PDzUo0cPRURE6MKFC5L+b9w9A8MVyGkDBw5URESEzGazDh06pMDAQE2cOFFpaWkaMGCABg4cqO+//94xzLt9+/bbtsOxibu1detWDRo0SJL0559/6syZM2revLkuX76sPXv2ZNq2Zs2akm5eHfuHH35QVFSUKlWqlOc13y8YVoNTHDx4UI888oi8vLwkSdHR0ZozZ47ee+89PfHEE0pJSdHMmTP1yy+/6PDhw2rRooX27dungQMHqnXr1k6uHvezw4cPKzU11TGJ+ocfftCgQYM0bNgwde7cWZK0Z88eLViwQCdPnlRycrLMZrPatGmj/v37O7N03GcOHTqk0NBQeXh4KDY2VqtXr5anp6cWLFig+fPna8yYMXrttdcc28fGxspkMsnV1TXT1ARkHz1HcIrt27dr0qRJkm52FderV0/e3t6aNWuWvv32W7m7u2vYsGEKCwvThAkTdOLECZ06dUq7du1ycuW43/3666+aPXu2Tpw4oc2bNyskJERNmjTRokWLtHjxYiUlJSkwMFDvv/++IiIiFBwcrPj4eI5N5LiaNWuqc+fO+uuvv1SyZEk9/PDD8vf3V7du3dStWzeNGzdOK1ascGz/wAMPyN/fn2CUA+g5Qp5LT0/XJ598oo0bNyotLU2nTp3SkSNHtHfvXk2dOlVms1kdO3bMdNPO+Ph4/fnnn6pduzbDFMhV3377rSIjI5WcnKyYmBgdOnRIFotFQ4YM0aFDh/T666+rbdu2KlasmGOf48ePq0KFCpwpiRyRcdZtamqqoqOjdfHiRX355Zfy8/PT7NmzVbx4cZ0/f17z58/XokWLNHz4cHXr1s3ZZd9X+E1GnnnttdcUHR0tFxcX9e/fXz4+Pjp16pRCQkIkSfXr19fQoUNls9m0bNkybdq0ybGvr6+v6tWr55h8DeSWFi1a6OGHH1ZMTIzq1aun3377TZI0depU1axZU4sWLdKaNWscJwxIUsWKFR2Tr4F7ZTabtX37ds2ePVvNmjXTgAEDFBYWpri4OPXp00dXrlyRv7+/evXqpX/961+KjIxUUlLSLXMxcfcIR8gTaWlpateunePKwZcvX5a/v786duyo2NhYvf3227p69arq1aunIUOGyG63a8mSJVq7du0tbdFzhJyW8aWSnp4uSapQoYLeeustpaSkKDIy0jFklhGQlixZoqVLlyolJSVTOxybyClHjhzRrFmztHLlSiUkJKh27dqaOnWq4uLi1KtXL50/f16HDx9W8+bNtWHDBnl7eztuuYR7x7Aa8lxkZKS8vLz0+uuvS7p5dsXGjRtVrVo1vffee/Ly8tJ//vMfjRw5Ug0bNtSYMWOcXDHuZ8YLh549e1be3t6OEwXWr1+vqKgolSpVSt27d1eDBg0kyTHxesaMGXwh4Z5l3ED2vy1YsEAffPCB+vTpo65du8rX11cHDx7U4MGDFR8fL3d3d0VHRysgIMAJVd/fuPgGcp3xFz8tLU0pKSmaPn260tLS1LNnT/Xu3Vsmk0nffvutJk6cqKFDh6po0aKKjIzkcvfIVcZgNHPmTK1bt04mk0l16tTRpEmT1KpVK5lMJs2dO1fz58/XlStXdP78eU2fPl3SzRt//t0XG5BVGcfP6dOnVaxYMZUoUUKS1K1bN9lsNk2ZMkWS1LVrV9WqVUvLly/X5s2b1ahRI4JRLqHnCLnq727n8cknn2jatGl688031atXL6WlpenTTz/V2rVr9ddff6l8+fLauHGjTCYTt11Arvvwww/1+eefq0+fPkpISNDq1avVrFkzTZw4UdLNm3tGRUUpJiZGrq6u+uGHH2SxWLhdDXKE3W7XqVOn9Oyzz6p///7q3LmzIyBJ0ty5cxUWFqZBgwbppZdeUpkyZZxXbCFBOEKuMX5xrF+/XseOHZPFYlHPnj0l3ewy/u+AtH//fsXExKhNmzZcVRh5Ytu2bZo0aZI+/PBD1apVS1u2bNGQIUNUrFgxBQUFKTw8XJL0+++/KyEhQYGBgQQj5Bhjz+P777+vpUuXauDAgXr11VcdAclms6lFixa6cOGCBg0apNdff51/H3MZ7y5yTcYXx7///W+tX79eZcqUUYMGDWQ2m+Xm5qb/+Z//kdVqVXh4uEwmk3r27KmGDRs69ud+VMgN/z0M9tdff8nLy0u1atXS0aNHtWTJEg0aNEiurq6aMmWKRo8ereHDhzvu6yeJ3kzcs4zj0Hgsjhw5UkWLFnUE8g4dOqh48eJKTU1Vw4YNVbZsWT3zzDP8u5gHeIeRq1atWqW1a9dq7ty5qlq1quLi4nTs2DHt2rVLISEhCg0NldlsVlhYmAICAjJd/Zp/AJDTjMEoLi5OpUqVkqenpypWrKhz585p3bp1qlSpkjp06KALFy4oLCxMK1euVHJyssLCwhztEIxwLzKOwz179ui7775TTEyMihUrpsGDB2vIkCFycXFReHi4UlJSFBgYqL179+rAgQMaMWKE42QB5C6G1ZCrIiIilJycrOHDh2v//v1avny5fvrpJ6Wnp+vGjRv68ssv9eCDD+rLL79U69atCUTINcZhsK+//lrbtm1T37595e/vrzNnzsjNzU19+/bVkCFD9PTTT+vIkSMKDw9X37599fjjjxOIkKM2bdqkkSNH6rnnnlOpUqW0cuVKFSlSRAsXLlTZsmU1Y8YMLViwQB4eHjKbzZo5c6Zq1Kjh7LILDb6JkGMy/jdk/N/5jRs3NG/ePF25ckXR0dFq3LixBg4cqKCgIPXo0UP79u1TuXLl1KZNG0kMpSF3GIPRTz/9pDVr1mj37t0ym83q3bu3KlasqB07dujatWsqU6aMbty4oYiICLm5ualOnTqcGIAcFRsbq/DwcA0cOFCvv/66zp8/r5UrV6p169a6fv26rl69qgEDBigkJEQmk0mlSpWSv7+/s8suVPgWQo4wfvlYrVZZrVYVKVJEgwcP1vXr13XkyBGNHTtWLVu2lJeXl65cuaLixYurePHimdohGCE3GOe/ffPNN2rUqJGeeOIJbdy4US4uLurataseeeQRXb58WaGhobJYLHJ3d9eqVatkMplks9kIRsgxGVez7ty5s86fP69XXnlFTZs2Vb9+/Rz3nBw3bpxq1arl5EoLL76JcM+MwWjhwoXasWOHzp49q3LlymnAgAEaMWKEJOnixYu6evWqLl26pPfff18Wi0WNGzd2ZukoRPbs2aOvv/5aERERqlmzpqSbZ1F+8sknunHjhkaNGqUvvvhC27dvl8Vi0WuvvSYXFxd6M5HjLBaLXFxctH79eoWHh6tJkyYaM2aMTCaTLl68yG1o8gF+43HPMoLR1KlT9fnnn6tDhw569NFHtXXrVvXo0UOTJk1SnTp1tGTJEs2ZM0ePPvqoPDw8tHz5cse90vhfOXJbamqqXFxcVKpUKccx16pVK9lsNg0fPlyS1K9fv0w38LRarQQj3JOMaQaXL19WkSJF5O7uLj8/P7m5uemtt95Sq1atNH78eMf2FotFpUuXzrQv8h4TspEjjh49qqFDh2r06NGO0/HT09PVq1cvnT59Wp999pmKFy+uLVu2yN/fX7Vq1ZLFYuF/5cgVt5v/9u2332ro0KH65ptvVK5cOV2/fl1FihTRjRs39Mwzz8hqtaply5b6n//5H/n5+Tn5FeB+kHH8bdmyRVFRUUpMTNQLL7ygTp066fTp0+rQoYMaNGigF154QQ8++KA2bNig6OhoLV++XBUrVnR2+YUaVzBDjjh//ryuXbumypUrS/q/idVTp05VamqqPvvsM7m5ually5aqU6eOo8eIYIScZrPZHIHIZrMpLS1NktSiRQvVrl1b/fr1U3x8vIoUKSLp5k2Qa9SooTZt2mjVqlU6dOiQ02rH/cVkMunHH3/U4MGD9dhjj6lKlSqaPXu2pkyZooceekhLlixRfHy8JkyYoBEjRmjv3r1atGgRwSgf4JsJ2Xa7rl43NzfduHFDR48e1RNPPCEXFxfZ7Xa5urrKz8/vtlcSZigNOc04/23ZsmXas2ePYmJiVLp0aQ0aNEhdu3bVrFmz9Morr2j48OGyWq1atWqVrl27ppkzZ+o///mPtm3bpqZNmzr5leB+cPnyZe3evVtDhw51DNcuX75cERERstvteuutt7R06VKdPXtWJpNJvr6+t5ykAucgHCFbjF8+qampslqt8vT0VJUqVeTp6anFixfLz89PlStXdlz9tUiRIipVqpSTK0dhkHFshoWFac2aNercubOaNm2qd999VwkJCfroo480ZswYRUVFafTo0fLz81PZsmW1YMECSVKRIkX0yCOPOPEV4H4RHx+vkJAQ+fj4aMCAAY7l//rXv2Sz2TRz5ky5uLioc+fO9BTlQ4QjZJndbnd8+cyePVv79+/X//7v/6px48bq0KGDZs6cqZdfflkpKSkKDg5W+fLltXTpUqWmpqp9+/ZOrh6FxW+//aZNmzZp6tSpCgwM1A8//CBXV1e99tprunDhgipWrKjp06crNjZWrq6ucnd3l6urq6ZPn64jR45o9OjRzn4JuA/4+vrqk08+0RtvvKGjR4/q8uXLjnuldezYURaLRRMmTFDRokU1ZMgQubm5ObdgZMKcI2SJcR7HJ598onnz5ikwMFAtWrTQ77//rv79+ysxMVGrV6+WxWLR3LlzNXv2bHl4eOiLL75wzDECctvFixclSYGBgfruu+80ePBgvfXWW2rZsqWmTZumWbNmSbr55RUbG6t27dqpQ4cOio6O1ty5c/XQQw85s3zcR5588klFRUVp6dKlWrNmjZKSkhzrOnTooLFjx+rVV18lGOVD9BwhSzJ6jE6cOKHjx49rzJgxev755yVJv/76qxYuXKhx48bp008/VUREhK5evSqLxSIfHx+ZTCbOSkOuuN38N29vb3l7e2vOnDmaNWuWRowYoQ4dOig5OVmJiYm6ceOGpJvz5KpXr66ePXvK399flSpVcpxCDeSUJ598Up9++qn69OkjSWrXrp28vb0lSS+//LIzS8M/oOcIWXbq1Ck9//zz2rBhQ6YvpBo1aqhjx466cuWK9u/fL3d3dz3wwAPy9fV1nE5NMEJOM/Zmnj59WvHx8ZKkBx98UFeuXFF4eLjeeOMNdejQQdLNq6+bTCZHALLZbJJufkEFBwcTjJBrgoODNXv2bIWHh+uzzz7TlStXnF0S7oBwhCwrX768FixY4DgrLSUlxbGuTp068vHx0R9//HHLflzEDDnNOP/t448/Vp8+ffTiiy9q6tSpcnd3V3h4uNzc3HTgwAHNmzdPGzduVO/evZWSkqIuXbpI0m3PoARyS3BwsKZNm6b58+czxaAA4F8HZEvDhg0VFRWlqKgofffdd7p+/bok6erVqzKbzSpXrpyTK8T9zthj9Omnn+qzzz5T586d1aJFCy1btkxTpkzRww8/rKVLl8put2vOnDmaO3euvLy8tGrVKrm4uPDlBKdo1qyZtmzZ4piYjfyLK2Tjrmzbtk19+/bV888/r3Llyum3337T2bNntXr1arm6ujq7PBQCp06d0vz589WkSROFhIRIunlto5kzZ6pZs2YaOnSoihcvrkuXLjH/DUC28C8E7krjxo316aefqkePHqpevbqaNm2q2bNnSxJfPsh1J0+e1LPPPit3d3c99dRTjuUdO3aUJEVGRspisahTp0569NFHHeuZ/wYgKxhWw1174oknNH/+fP3+++/y8vJynAXElw9y20MPPaQFCxYoJSVFBw4cUEJCgmNdx44d1b9/fy1fvlzbtm3LtB/z3wBkBcNquGfbt29XaGio+vXrp06dOqlYsWLOLgmFxE8//aSePXvq7bffznSKtCRt3rxZTZs25TY1ALKNcIQcsWXLFr3zzjvasGEDkw2Rp7Zv364+ffpo+PDhtwQkSbJarQQkANlCOEKOSU5OloeHh7PLQCH0448/qm/fvurbt6+6du0qT09PZ5cEoABjzhFyDMEIzvLUU09p2rRp+umnnzgOAdwzeo4A3Dcybidyu9uKAEBW0XME4L5BMAKQEwhHAO4rBCMA94pwBAAAYEA4AgAAMCAcAQAAGBCOAOD/4+RdABLhCEAB1aVLF1WpUiXTn6pVq6pu3bpq166dvvzyy2y1t2/fPvXu3dvxOCYmRlWqVNHq1atzunQA+Rx3CAVQYFWrVk1jxoxxPLZarYqNjdWCBQs0fPhwlShRQiEhIVlq6/PPP9fx48cdj0uVKqUVK1aofPnyOV43gPyNcASgwPLy8lLt2rVvWd64cWM1atRIq1evznI4+m9ubm63bRvA/Y9hNQD3nSJFisjNzc1xzaP4+HiNGzdOTZs2VY0aNRQUFKTQ0FDFxMRIkkaMGKHo6GidOXPGMZT238Nqq1evVrVq1fTLL7+oQ4cOqlmzppo2baqoqKhMzx0XF6chQ4YoKChIgYGBGj16tKZOnapmzZrl7ZsA4K7RcwSgwLLb7UpPT3c8tlqtOnPmjGbOnKlr167ppZdekt1uV58+fZSYmKhhw4bJz89Pv//+u6ZNm6YxY8YoKipK/fr1U3x8vA4fPqyIiAiVL19eycnJtzyfzWbT4MGD1a1bNw0ePFhffPGFpkyZosqVKys4OFhpaWnq2rWrkpOT9c4778jLy0tz5szRkSNH9MADD+TlWwPgHhCOABRYe/bsUfXq1TMtM5lMqly5sj7++GM1bdpU58+fl7u7u95++23Vr19fktSgQQOdOnVKK1askCSVL19evr6+mYbSbheO7Ha7+vXrp1deeUWSVK9ePW3atElbt25VcHCwvvrqK504cUKrVq1SjRo1JEkNGzbUM888k1tvAYBcQDgCUGBVr15d48aNk3RzOGvatGm6ceOGpk2bpkceeUSS5O/vr0WLFslutysmJkYnT57UiRMntH//fqWlpWX7OevUqeP42c3NTb6+vo4gtXPnTpUrV84RjKSb86KaNm2qXbt23ctLBZCHCEcACixPT0/VrFnT8fjxxx/Xiy++qB49emj16tXy9fWVJH311VcKDw/XuXPnVKJECT322GMqWrToXT3nf+9nNpsd10dKSEhQyZIlb9nndssA5F9MyAZw3/Dz89Po0aN17tw5TZo0SZK0d+9evf3222rRooW2bdumXbt2acGCBblyJpq/v78uXrx4y/JLly7l+HMByD2EIwD3lZYtWyo4OFjr1q3T7t27deDAAdlsNg0YMED+/v6Sbk7c/vnnnyXdnGQt3ewBuldBQUGKiYnRkSNHHMtSU1O1ffv2e24bQN4hHAG477zzzjtydXXVxIkTHfN/xo8fr507d2rjxo3q3r27jh49Kun/Jl57e3vr4sWL+uGHHxQXF3dXz9u6dWtVrFhRoaGh+vLLL/X999+rd+/eunTpkuOyAgDyP8IRgPvOI488oi5duuj333/X8ePHNXr0aB04cEC9evXSBx98oDJlyigiIkLSzduGSFK7du1UtmxZhYaGas2aNXf1vC4uLoqKilK1atU0duxYDR8+XI8++qiaN28uDw+PnHp5AHKZyc6dFgEgR/zxxx86ceKEWrRokamn6OWXX1ZAQIAjkAHI3zhbDQBySHJysgYNGqSOHTuqefPmslqt+vrrr/Xrr79q2LBhzi4PQBbRcwQAOWjDhg2KiorS8ePHZbfbVa1aNfXt21dPPfWUs0sDkEWEIwAAAAMmZAMAABgQjgAAAAwIRwAAAAaEIwAAAAPCEQAAgAHhCAAAwIBwBAAAYEA4AgAAMCAcAQAAGPw/KDv6LNLwEVUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the value counts as a bar plot\n",
    "df['categorical_rating'].value_counts().plot(kind='bar')\n",
    "\n",
    "# Set the title and labels\n",
    "plt.title('Rating Distribution')\n",
    "plt.xlabel('Rating')\n",
    "plt.ylabel('No. of Reviews')\n",
    "\n",
    "categories = df['categorical_rating'].value_counts().index  # Get the categories (which are the index of the value_counts Series)\n",
    "custom_labels = [cat for cat in categories]  # Create custom labels\n",
    "plt.xticks(ticks=range(len(categories)), labels=custom_labels, rotation=45)  # Set custom labels with rotation\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42c0a249-c05d-4c1c-9060-9ccf1e8490b7",
   "metadata": {},
   "source": [
    "## Preparing Data for Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8c348fd8-a746-4363-96d6-be43cea1203d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: Create X and y\n",
    "X = df['review_text']\n",
    "y = pd.get_dummies(df['categorical_rating'])  # Step 2: One-hot encode y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b3dc54f7-25a2-44cf-ad63-2dbe4dcc5c70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initial split: 80% for training, 20% for temporary dataset\n",
    "X_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Split the temporary dataset equally into validation and test sets: 10% each of the original dataset\n",
    "X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5b4c0efc-126f-465c-b266-975f12a401c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Chetouani\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\tokenization_utils_base.py:2645: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Load the BERT tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Tokenize and prepare the data for BERT\n",
    "def encode_reviews(tokenizer, reviews, max_length):\n",
    "    return tokenizer.batch_encode_plus(\n",
    "        reviews,\n",
    "        add_special_tokens=True,\n",
    "        return_attention_mask=True,\n",
    "        pad_to_max_length=True,\n",
    "        max_length=max_length,\n",
    "        truncation = True,\n",
    "        return_tensors='tf',\n",
    "    )\n",
    "\n",
    "# Choose a maximum sequence length for BERT\n",
    "max_length = 256\n",
    "\n",
    "# Encode the datasets\n",
    "train_encodings = encode_reviews(tokenizer, X_train.tolist(), max_length)\n",
    "val_encodings = encode_reviews(tokenizer, X_val.tolist(), max_length)\n",
    "test_encodings = encode_reviews(tokenizer, X_test.tolist(), max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6e432b5c-263b-484a-aaec-1e834dbc89d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing TFBertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_ids (InputLayer)         [(None, 256)]        0           []                               \n",
      "                                                                                                  \n",
      " attention_masks (InputLayer)   [(None, 256)]        0           []                               \n",
      "                                                                                                  \n",
      " tf_bert_model (TFBertModel)    TFBaseModelOutputWi  109482240   ['input_ids[0][0]',              \n",
      "                                thPoolingAndCrossAt               'attention_masks[0][0]']        \n",
      "                                tentions(last_hidde                                               \n",
      "                                n_state=(None, 256,                                               \n",
      "                                 768),                                                            \n",
      "                                 pooler_output=(Non                                               \n",
      "                                e, 768),                                                          \n",
      "                                 past_key_values=No                                               \n",
      "                                ne, hidden_states=N                                               \n",
      "                                one, attentions=Non                                               \n",
      "                                e, cross_attentions                                               \n",
      "                                =None)                                                            \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem (Slic  (None, 768)         0           ['tf_bert_model[0][0]']          \n",
      " ingOpLambda)                                                                                     \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 512)          393728      ['tf.__operators__.getitem[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " dropout_37 (Dropout)           (None, 512)          0           ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 256)          131328      ['dropout_37[0][0]']             \n",
      "                                                                                                  \n",
      " dense_2 (Dense)                (None, 3)            771         ['dense_1[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 110,008,067\n",
      "Trainable params: 110,008,067\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from transformers import TFBertModel\n",
    "from tensorflow.keras.layers import Input, Dense, Dropout\n",
    "from tensorflow.keras.models import Model\n",
    "\n",
    "\n",
    "# Load the pre-trained BERT model\n",
    "bert = TFBertModel.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Build the model\n",
    "input_ids = Input(shape=(max_length,), dtype='int32', name='input_ids')\n",
    "attention_masks = Input(shape=(max_length,), dtype='int32', name='attention_masks')\n",
    "\n",
    "# Get the sequence output\n",
    "sequence_output = bert(input_ids, attention_mask=attention_masks)[0]\n",
    "\n",
    "# Select the first token's last hidden state\n",
    "cls_token = sequence_output[:, 0, :]\n",
    "\n",
    "# Add custom layers\n",
    "x = Dense(512, activation='relu')(cls_token)\n",
    "x = Dropout(0.1)(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "output = Dense(y.shape[1], activation='softmax')(x)\n",
    "\n",
    "# Compile the model\n",
    "model = Model(inputs=[input_ids, attention_masks], outputs=output)\n",
    "\n",
    "# Adjust the learning rate\n",
    "optimizer = Adam(learning_rate=2e-5)\n",
    "\n",
    "# Compile the model with the new optimizer\n",
    "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Summary of the model\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aef868b5-8991-44bf-a7f2-1b1e1963078e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "251/251 [==============================] - 126s 355ms/step - loss: 0.2646 - accuracy: 0.9089 - val_loss: 0.2234 - val_accuracy: 0.9281\n",
      "Epoch 2/10\n",
      "251/251 [==============================] - 87s 344ms/step - loss: 0.1750 - accuracy: 0.9449 - val_loss: 0.1983 - val_accuracy: 0.9301\n",
      "Epoch 3/10\n",
      "251/251 [==============================] - 82s 327ms/step - loss: 0.1266 - accuracy: 0.9553 - val_loss: 0.2145 - val_accuracy: 0.9301\n",
      "Epoch 4/10\n",
      "251/251 [==============================] - 82s 329ms/step - loss: 0.0898 - accuracy: 0.9731 - val_loss: 0.2907 - val_accuracy: 0.9242\n",
      "Epoch 5/10\n",
      "251/251 [==============================] - 83s 332ms/step - loss: 0.0754 - accuracy: 0.9763 - val_loss: 0.2802 - val_accuracy: 0.9222\n",
      "Epoch 6/10\n",
      "251/251 [==============================] - 83s 329ms/step - loss: 0.0509 - accuracy: 0.9838 - val_loss: 0.5408 - val_accuracy: 0.9062\n",
      "Epoch 7/10\n",
      "251/251 [==============================] - 83s 329ms/step - loss: 0.0322 - accuracy: 0.9913 - val_loss: 0.3736 - val_accuracy: 0.9182\n",
      "Epoch 8/10\n",
      "251/251 [==============================] - 75s 301ms/step - loss: 0.0318 - accuracy: 0.9908 - val_loss: 0.3482 - val_accuracy: 0.9162\n",
      "Epoch 9/10\n",
      "251/251 [==============================] - 75s 300ms/step - loss: 0.0347 - accuracy: 0.9908 - val_loss: 0.4012 - val_accuracy: 0.9102\n",
      "Epoch 10/10\n",
      "251/251 [==============================] - 77s 305ms/step - loss: 0.0228 - accuracy: 0.9938 - val_loss: 0.4097 - val_accuracy: 0.9222\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "\n",
    "model_save_path = r\"D:\\Marketing Paper\\best_model_apple.h5\"\n",
    "\n",
    "checkpoint = ModelCheckpoint(model_save_path, monitor='val_accuracy', save_best_only=True)\n",
    "\n",
    "# Train the model\n",
    "history = model.fit(\n",
    "    [train_encodings['input_ids'], train_encodings['attention_mask']],\n",
    "    y_train,\n",
    "    validation_data=([\n",
    "        val_encodings['input_ids'], val_encodings['attention_mask']\n",
    "    ], y_val),\n",
    "    epochs=10,\n",
    "    batch_size=16,  # Specify your desired batch size here\n",
    "    callbacks=[checkpoint]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "202e256d-e01c-4ba4-ba68-3f5cd4f77aaa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n",
      "16/16 [==============================] - 10s 196ms/step - loss: 0.2490 - accuracy: 0.9301\n",
      "Test Loss: 0.24903960525989532, Test Accuracy: 0.9301397204399109\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "# Load the best model\n",
    "best_model = load_model(model_save_path, custom_objects={'TFBertModel': TFBertModel})\n",
    "\n",
    "# # Recompile the model with the optimizer, loss, and metrics\n",
    "# model.compile(optimizer=Adam(learning_rate=2e-5), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Evaluate the model on the test set\n",
    "test_loss, test_acc = best_model.evaluate(\n",
    "    [test_encodings['input_ids'], test_encodings['attention_mask']],\n",
    "    y_test\n",
    ")\n",
    "\n",
    "print(f\"Test Loss: {test_loss}, Test Accuracy: {test_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e1c534b9-45bc-471e-998b-c0305801341e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16/16 [==============================] - 3s 184ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.73      0.72      0.73        46\n",
      "     neutral       0.00      0.00      0.00        19\n",
      "    positive       0.95      0.99      0.97       436\n",
      "\n",
      "    accuracy                           0.93       501\n",
      "   macro avg       0.56      0.57      0.57       501\n",
      "weighted avg       0.89      0.93      0.91       501\n",
      "\n",
      "Accuracy for class negative: 0.72\n",
      "Accuracy for class neutral: 0.00\n",
      "Accuracy for class positive: 0.99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Chetouani\\anaconda3\\envs\\py310\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1509: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "C:\\Users\\Chetouani\\anaconda3\\envs\\py310\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1509: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "C:\\Users\\Chetouani\\anaconda3\\envs\\py310\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1509: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "# Assuming `test_encodings` is already prepared similar to `train_encodings` and `val_encodings`\n",
    "predictions = best_model.predict([test_encodings['input_ids'], test_encodings['attention_mask']])\n",
    "\n",
    "# Assuming you have predictions from the model\n",
    "predicted_class_indices = predictions.argmax(axis=1)\n",
    "\n",
    "class_names = y_test.columns.tolist() \n",
    "\n",
    "# Convert y_test from one-hot encoding to class indices\n",
    "true_class_indices = y_test.idxmax(axis=1).apply(class_names.index).values\n",
    "\n",
    "\n",
    "# Generate the classification report using the true and predicted class indices\n",
    "report = classification_report(true_class_indices, predicted_class_indices, target_names=class_names)\n",
    "print(report)\n",
    "\n",
    "# Generate the confusion matrix\n",
    "cm = confusion_matrix(true_class_indices, predicted_class_indices)\n",
    "\n",
    "# Calculate accuracy for each class\n",
    "class_accuracies = cm.diagonal() / cm.sum(axis=1)\n",
    "for class_name, accuracy in zip(class_names, class_accuracies):\n",
    "    print(f'Accuracy for class {class_name}: {accuracy:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba7db7ff-c2b6-40ae-bcca-cee9cc37a2b0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
